package com.uusee.crawler.job;

import java.util.ArrayList;
import java.util.List;

import org.apache.commons.lang.time.StopWatch;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.oro.text.regex.MatchResult;
import org.apache.oro.text.regex.Pattern;
import org.apache.oro.text.regex.PatternCompiler;
import org.apache.oro.text.regex.PatternMatcher;
import org.apache.oro.text.regex.PatternMatcherInput;
import org.apache.oro.text.regex.Perl5Compiler;
import org.apache.oro.text.regex.Perl5Matcher;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;

import com.uusee.crawler.dbwriter.EpisodeDbWriter;
import com.uusee.crawler.fetcher.FetchHTTP;
import com.uusee.crawler.model.CrawlStatusCodes;
import com.uusee.crawler.model.CrawlURI;
import com.uusee.crawler.pageprocessor.baike.mtime.EpisodePageProcessor;
import com.uusee.framework.bo.UniversalBo;
import com.uusee.framework.util.query.CriteriaInfo;
import com.uusee.framework.util.query.Sort;
import com.uusee.shipshape.bk.Constants;
import com.uusee.shipshape.bk.model.Baike;
import com.uusee.shipshape.bk.model.Episode;

public class MtimeEpisodeCrawlJob {
	private Log log = LogFactory.getLog(MtimeImagesCrawlJob.class);
	private FetchHTTP fetch = new FetchHTTP();
	private EpisodePageProcessor pageProcessor = new EpisodePageProcessor();
	private EpisodeDbWriter dbWriter = new EpisodeDbWriter();
	private String sourceSite = "www.mtime.com";
	private UniversalBo universalBo;
	
	public static void main(String[] args) {
		try {
			ApplicationContext acx = new ClassPathXmlApplicationContext("spring/applicationContext-framework.xml");
			MtimeEpisodeCrawlJob job = new MtimeEpisodeCrawlJob();
			UniversalBo universalBo = (UniversalBo) acx.getBean("universalBo");
			job.dbWriter.setUniversalBo(universalBo);
			job.universalBo = universalBo;
			job.doExecute();
		} catch (Exception e) {
			e.printStackTrace();
		}
	}

	public void doExecute() {
		try {
			StopWatch stopWatch = new StopWatch();
			stopWatch.start();
			//2009年9月16日：完成181979
			long startId = 181979l;
			CriteriaInfo ci = new CriteriaInfo();
			ci.eq("sourceSite", Constants.SOURCE_SITE_MTIME);
			ci.eq("channelCode", Constants.CHANNEL_CODE_TELEPLAY);
			//ci.ne("videoStatus", Constants.VIDEO_STATUS_DEFAULT);
			
			ci.le("id", startId);
			int total = universalBo.getCountByCriteriaInfo(Baike.class, ci);
			
			int maxResult = 500;
			int start = 0;
			while(start<total){
				ci = new CriteriaInfo();
				ci.eq("sourceSite", Constants.SOURCE_SITE_MTIME);
				ci.eq("channelCode", Constants.CHANNEL_CODE_TELEPLAY);
				//ci.ne("videoStatus", Constants.VIDEO_STATUS_DEFAULT);
				ci.le("id", startId);
				ci.addSort("id", Sort.DESC);
				ci.setOffset(true);
				ci.setFirstResult(start);
				ci.setMaxResult(maxResult);
				List<Baike> baikeList = universalBo.getEntitiesByCriteriaInfo(Baike.class, ci);
				log.info("百科数："+baikeList.size());
				int i=0;
				for (Baike bk:baikeList) {
					log.info("抓取百科："+bk.getId()+"-"+bk.getOriId());
					crawl(bk.getOriId());
					i++;
					if (i%50==0) {
						log.info("抓取百科数："+i+";用时:" + stopWatch.getTime() / 1000 + "秒");
					}
					
				}
				log.info(sourceSite + "-抓取电影数:" + start + ";用时:" + stopWatch.getTime() / 1000 + "秒");
				start=start+maxResult;
			}
			stopWatch.stop();
			log.info("抓取结束,用时：" + stopWatch.getTime() / 1000 + "秒。");
		} catch (Exception e) {
			log.error(e);
		}
	}

	public CrawlURI createCrawlURI(String mtimeMovieId) {
		CrawlURI crawlURI = new CrawlURI();
		String crawlUrl = "http://www.mtime.com/movie/" + mtimeMovieId + "/episodeplots.html";
		crawlURI.setSourceSite("www.mtime.com");
		crawlURI.setCrawlUrl(crawlUrl);
		crawlURI.setCharset("utf-8");

		List<Episode> episodeList = new ArrayList<Episode>();
		crawlURI.setModel(episodeList);

		return crawlURI;
	}

	public void crawl(String movieId) {
		try {
			String crawlUrl = "http://www.mtime.com/movie/" + movieId + "/episodeplots.html";
			CrawlURI cURI = new CrawlURI();
			cURI.setSourceSite("www.mtime.com");
			cURI.setCrawlUrl(crawlUrl);
			cURI.setCharset("utf-8");
			List<CrawlURI> crawlURIList = new ArrayList<CrawlURI>();
			cURI.setModel(crawlURIList);
			getUrlList(cURI);
			List<CrawlURI> episodecrawlURIList = (List<CrawlURI>) cURI.getModel();
			for (CrawlURI cu : episodecrawlURIList) {
				crawl(cu);
			}
		} catch (Exception e) {
			e.printStackTrace();
			log.error(movieId+"抓取分集剧情失败!", e);
		}
	}

	public void crawl(CrawlURI crawlURI) {
		try {
			fetch.process(crawlURI);
			pageProcessor.process(crawlURI);
			if (crawlURI.getCrawlStatus() == CrawlStatusCodes.PAGE_PROCESS_SUCCESS) {
				List<CrawlURI> episodeCrawlURIList = (List<CrawlURI>) crawlURI.getModel();
				for (CrawlURI episodeCrawlURI : episodeCrawlURIList) {
					episodeCrawlURI.setCrawlStatus(CrawlStatusCodes.PAGE_PROCESS_SUCCESS);
					dbWriter.process(episodeCrawlURI);
				}
			}
		} catch (Exception e) {
			e.printStackTrace();
		}
	}

	private void getUrlList(CrawlURI crawlURI) {
		List<CrawlURI> episodecrawlURIList = (List<CrawlURI>) crawlURI.getModel();
		try {
			fetch.process(crawlURI);
			String crawlResult = crawlURI.getCrawlResult();
			if (crawlResult.indexOf("<div id=\"movie_main_l\"") < 0) {
				crawlURI.setCrawlStatus(CrawlStatusCodes.PAGE_PROCESS_INVALID);
				return;
			}
			crawlURI.setCrawlStatus(CrawlStatusCodes.PAGE_PROCESS_SUCCESS);
			String regx = "<dl class=\"ele_img_item lh18\">\\s*?<dt class=\"ele_img_box pr12\">[^<]*?</dt><dd class=\"ele_img_content\">\\s*?<a href=\"(.*?)#[^<]*?>[^<]*?</a>";
			PatternCompiler compiler = new Perl5Compiler();
			Pattern pattern = compiler.compile(regx, Perl5Compiler.CASE_INSENSITIVE_MASK);
			PatternMatcher matcher = new Perl5Matcher();
			PatternMatcherInput input = new PatternMatcherInput(crawlResult);
			while (matcher.contains(input, pattern)) {
				MatchResult m = matcher.getMatch();
				String crawlUrl = m.group(1);
				if (!crawlUrl.startsWith("http://")) {
					crawlUrl = "http://www.mtime.com" + crawlUrl;
				}
				CrawlURI episodecrawlURI = new CrawlURI();
				episodecrawlURI.setSourceSite("www.mtime.com");
				episodecrawlURI.setCrawlUrl(crawlUrl);
				episodecrawlURI.setCharset("utf-8");

				List<Episode> episodeList = new ArrayList<Episode>();
				episodecrawlURI.setModel(episodeList);
				episodecrawlURIList.add(episodecrawlURI);
			}
		} catch (Exception e) {
			crawlURI.setCrawlStatus(CrawlStatusCodes.PAGE_PROCESS_EXCEPTION);
			log.error("getUrlList失败!", e);
		}
	}

	public void setFetch(FetchHTTP fetch) {
		this.fetch = fetch;
	}

	public void setPageProcessor(EpisodePageProcessor pageProcessor) {
		this.pageProcessor = pageProcessor;
	}

	public void setDbWriter(EpisodeDbWriter dbWriter) {
		this.dbWriter = dbWriter;
	}

	public void setSourceSite(String sourceSite) {
		this.sourceSite = sourceSite;
	}

	public void setUniversalBo(UniversalBo universalBo) {
		this.universalBo = universalBo;
	}
}
